// #include "../include/WordSegmentation.h"
#include "../../include/WebPage.h"
#include "../../include/PageLib.h"
#include <ctype.h>

// WordSegmentation wordSeg;
WebPage::WebPage(const RssItem &rss)
:_docId(0), _docTitle(rss.title), _docUrl(rss.url), _docContent(rss.content), _totalWordsFreq(0)
{
    // cout << "WebPage(rss)" << endl;
    // cout <<  _docId << endl;
    // cout << _docTitle << endl;
    // cout << _docUrl << endl;
    // cout << _docContent << endl;
}

void WebPage::createWordsFreq(const unordered_set<string>& stopWords, WordSegmentation &wordSeg)
{

    vector<string> words = wordSeg.cutforSearch(_docContent);

    for (auto &word : words) {
        if(!stopWords.count(word)) {
            // if(isalpha(word[0])) {
            //     dealWord(word);
            // }
            ++_wordsFreq[word];
            ++_totalWordsFreq;
        }
    }
}
//英文预处理
void WebPage::dealWord(string& word) {
    for (auto& ch : word) {
        if (!isalpha(ch)) {
            ch = ' ';
        } else if (isupper(ch)) {
            ch = tolower(ch);
        }
    }
}